package homework9package;


import redis.clients.jedis.*;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.io.*;
import java.net.URL;


/**
 * Indexer class
 * 
 * 
 * this class provides functions for storing objects in a redis database
 * 
 * 
 * @author Joe Gibson and Orion Taylor
 * @course Software Engineering
 * @date 4-1-13
 */

public class Indexer {
	
	public static void main (String [] args)
			throws ClassNotFoundException, IOException{
		
		
		
		//create a urlread object associated with "http://www.serebii.net"
		urlreader q = new urlreader("http://www.serebii.net");
		
		//create a urlread object associated with "http://google.com"
				urlreader r = new urlreader("http://www.google.com");
		 
		Indexer my_Indexer=new Indexer();
		ArrayList  <urlreader> url_list =new ArrayList<urlreader>();
		url_list.add(q);
		url_list.add(r);
		
		my_Indexer.addurlreader(url_list);
		
		Index myIndex=my_Indexer.retrieveDatabase();
		
		//url_index maps url's to their integer codes
		HashMap<String,Integer> url_index=
				myIndex.getUrl_index();
		
		//term_index maps terms to their integer codes
		HashMap<String,Integer> term_index=
				myIndex.getTerm_index();
		
		//forward_edge_map maps urls to the url's they are linked to	
		HashMap<Integer,HashSet<Integer>> forward_edge_map=
				myIndex.getForward_edge_map();
		
		//backward_edge_maps maps urls to the url's that link to them
		HashMap<Integer,HashSet<Integer>> backward_edge_map=
				myIndex.getBackward_edge_map();
		
		//forward_term_map maps urls to the terms on their page
		//and their corresponding pair frequencies
		HashMap<Integer,HashMap<Integer,Integer>> forward_term_map=
				myIndex.getForward_term_map();
		
		//backward_term_map maps terms to the urls that have them
		//and their corresponding pair frequencies
		HashMap<Integer,HashMap<Integer,Integer>> backward_term_map=
				myIndex.getBackward_term_map();
		
		System.out.println(url_index);
		System.out.println(term_index);
		System.out.println(forward_edge_map);
		System.out.println(backward_edge_map);
		System.out.println(forward_term_map);
		System.out.println(backward_term_map);
	}
	
	
	/** function initialize_Database()
	 * 
	 * this function initializes all of the persistent objects
	 * in our prespecified redis database. If these
	 * object already exist, then they are reset to their
	 * initial values
	 * 
	 * @param none: this function takes in no objects
	 * @return void: this function has no returns
	 * @throws IOException 
	 * @throws ClassNotFoundException 
	 */
	public void initialize_Database()
			throws ClassNotFoundException, IOException{
		
		
		//url_index maps url's to their integer codes
		this.storeObject("url_index",new HashMap<String,Integer>());
		
		//term_index maps search terms to their integer codes
		this.storeObject("term_index", new HashMap<String,Integer> ());
		
		//forward_edge_map maps urls to the url's they are linked to
		this.storeObject("forward_edge_map",
				new HashMap<Integer,HashSet<Integer>> ());
		
		//backward_edge_maps maps urls to the url's that link to them
		this.storeObject("backward_edge_map", 
				new HashMap<Integer,HashSet<Integer>> ());
		
		//forward_term_map maps urls to the terms on their page
		//and their corresponding pair frequencies
		this.storeObject("forward_term_map",
				new HashMap<Integer,HashMap<Integer,Integer>> ());
				
		//backward_term_map maps terms to the urls that have them
		//and their corresponding pair frequencies
		this.storeObject("backward_term_map", 
				new HashMap<Integer,HashMap<Integer,Integer>> ());
		
	}
	
	/**retrieveDatabasae()
	 * 
	 * this is a getter that retrieves the
	 * various persistent objects on the database
	 * not much more to say here...
	 * @return Index: a representation of the objects in our indexer database
	 * @throws IOException 
	 * @throws ClassNotFoundException 
	 */
	@SuppressWarnings("unchecked")
	public Index retrieveDatabase()
			throws ClassNotFoundException, IOException{
		
		//myIndex represents the web index we are downloading
		Index myIndex=new Index();
		
		//retrieve each corresponding field from the database
		//then store it in our web index
		
		myIndex.setUrl_index((HashMap<String,Integer>) 
				this.retrieveObject("url_index"));
		
		myIndex.setTerm_index((HashMap<String,Integer>) 
				this.retrieveObject("term_index"));
		
		myIndex.setForward_edge_map((HashMap<Integer,HashSet<Integer>>) 
				this.retrieveObject("forward_edge_map"));
		
		myIndex.setBackward_edge_map((HashMap<Integer,HashSet<Integer>>) 
				this.retrieveObject("backward_edge_map"));
		
		myIndex.setForward_term_map((HashMap<Integer,HashMap<Integer,Integer>>) 
				this.retrieveObject("forward_term_map"));
		
		myIndex.setBackward_term_map((HashMap<Integer,HashMap<Integer,Integer>>) 
				this.retrieveObject("backward_term_map"));
		
		return myIndex;
		
	}
	
	/**storeDatabasae()
	 * 
	 * this is a setter that stores the
	 * various persistent objects that represent the web on the database
	 * not much more to say here...
	 * @param Index: a representation of the objects in our indexer database
	 * @throws IOException 
	 * @throws ClassNotFoundException 
	 */
	private void storeDatabasae(Index myIndex)
			throws ClassNotFoundException, IOException{
		
		
		//retrieve each corresponding field from the web index
		//then store it in our online database
		
		//store updated url_index
		this.storeObject("url_index",myIndex.getUrl_index());
		
		//store updated term_index
		this.storeObject("term_index", myIndex.getTerm_index());
		
		//store updated  forward_edge_map
		this.storeObject("forward_edge_map", myIndex.getForward_edge_map());
		
		//store updated backward_edge_maps
		this.storeObject("backward_edge_map", myIndex.getBackward_edge_map());
		
		//store updated forward_term_map
		this.storeObject("forward_term_map", myIndex.getForward_term_map());
				
		//store updated  backward_term_map
		this.storeObject("backward_term_map", myIndex.getBackward_term_map());
			
	}
	
	
	/** function addurlreader
	 * 
	 * this function takes in an ArrayList urlreader objects
	 * and adds all of its data to the online database
	 * in a useful and constructive manner.
	 * 
	 * 
	 * @param my_readers: an ArrayList of the urlreader objects
	 * we want to store in the database
	 * @return void: this function has no returns
	 * @throws IOException 
	 * @throws ClassNotFoundException 
	 */
	private void addurlreader(ArrayList<urlreader> my_readers)
			throws ClassNotFoundException, IOException{
		
		//retrieve a representation of the database and store it in index
		Index myIndex= this.retrieveDatabase();
		
		//iterate through each urlreader in my_reader
		for(urlreader my_reader:my_readers){
			
			//add the current urlreader to the representation of the database
			this.addurlreader(my_reader, myIndex);
		}
		
		//store the representation of the database back into the database
		this.storeDatabasae(myIndex);
		
	}
	
	/** function addurlreader
	 * 
	 * this function takes in a urlreader object
	 * and adds all of its data to a representation of
	 * the database in a useful and constructive manner.
	 * 
	 * 
	 * @param my_reader: the urlreader object we want to store in the database
	 * @param myIndex: a representation of the database that we want to
	 * store our urlreader into
	 * @return void: this function has no returns
	 */
	private void addurlreader(urlreader my_reader, Index myIndex){
		
		//local_URL represents the URL that this urlreader object actually loaded
		String local_URL = my_reader.getlocal_URL();
		
		//URL_list is a list of the URL's on the webpage associated with the URL
		ArrayList<URL> URL_list= my_reader.getURL_list();
		
		//word_freq is a HashMap that maps a word in the page to
		//its corresponding frequency
		HashMap<String,Integer> word_freq =my_reader.getword_freq();
		
		
		//make sure that the url visited is in the url index
		this.addURL_toIndex(local_URL,myIndex);
		
		
		//iterate through all the linked to URL's
		for(URL this_URL: URL_list){
			//store each one in the url index
			this.addURL_toIndex(this_URL.toString(),myIndex);
		}
		
		//iterate through all the terms on the page
		for(String term: word_freq.keySet()){
			//store each one in the term index
			this.addTerm_toIndex(term,myIndex);
		}
		
		//url_index maps url's to their integer codes
		HashMap<String, Integer> url_index = myIndex.getUrl_index();
	
		//term_index maps terms to their integer codes
		HashMap<String, Integer> term_index = myIndex.getTerm_index();
		
		//forward_edge_map maps urls to the url's they are linked to
		HashMap<Integer, HashSet<Integer>> forward_edge_map =
				myIndex.getForward_edge_map();
		
		//backward_edge_maps maps urls to the url's that link to them
		HashMap<Integer, HashSet<Integer>> backward_edge_map =
				myIndex.getBackward_edge_map();
		
		//forward_term_map maps urls to the terms on their page
		//and their corresponding pair frequencies
		HashMap<Integer, HashMap<Integer, Integer>> forward_term_map =
				myIndex.getForward_term_map();
		
		//backward_term_map maps terms to the urls that have them
		//and their corresponding pair frequencies
		HashMap<Integer, HashMap<Integer, Integer>> backward_term_map =
				myIndex.getBackward_term_map();
		
		
		//get the code corresponding to the local URL
		Integer local_code=url_index.get(local_URL);
		
		
		//the nexus URL is not in the forward edge map
		if(!forward_edge_map.containsKey(local_code)){
			
			//put a new key-value pair in the forward_edge_map
			forward_edge_map.put(local_code, new HashSet<Integer>());
		}
		
		//the nexus URL is not in the forward edge map
		if(!forward_term_map.containsKey(local_code)){
			
			//put a new key-value pair in the forward_term_map
			forward_term_map.put(local_code, new HashMap<Integer,Integer>());
		}
		
				
		//iterate through all the linked to URL's
		for(URL this_URL: URL_list){
			
			Integer other_url_code=url_index.get(this_URL.toString());
			
			//add the to forward edge map
			forward_edge_map.get(local_code).add(other_url_code);
			
			
			//the other URL is not in the backward edge map
			if(!backward_edge_map.containsKey(other_url_code)){
				
				//put a new key-value pair in the backward_edge_map
				backward_edge_map.put(other_url_code, new HashSet<Integer>());
			}
			
			//add to backward edge map
			backward_edge_map.get(other_url_code).add(local_code);
			
		}
		
		//iterate through all the terms on the page
		for(String term: word_freq.keySet()){
			
			Integer term_code=term_index.get(term);
			
			//add the to forward edge map
			forward_term_map.get(local_code).put(term_code, word_freq.get(term));
			
			
			//the other URL is not in the backward edge map
			if(!backward_term_map.containsKey(term_code)){
				
				//put a new key-value pair in the backward_edge_map
				backward_term_map.put(term_code, new HashMap<Integer,Integer>());
			}
			
			//add to backward edge map
			backward_term_map.get(term_code).put(local_code, word_freq.get(term));
		}
		
	}
	
	
	/** function addURL_toIndex
	 * 
	 * this function takes in a string representing a URL as input
	 * it then adds the URL to url_index (with a corresponding code)
	 * note: (url_index is a field in the myIndex)
	 * its simply a mapping of urls to numbers (which are unique)
	 * 
	 * if the URL is already in the index, this function does nothing
	 * 
	 * 
	 * @param local_URL: the URL we want to store
	 * @param myIndex: a representation of the web database
	 * @return void: this function has no returns
	 */
	private void addURL_toIndex(String local_URL, Index myIndex){
	
		HashMap<String, Integer> url_index = myIndex.getUrl_index();
		
		//if the index does not already contain the URL in question
		if(!url_index.containsKey(local_URL)){
			//add the URL to the index
			url_index.put(local_URL, url_index.size()+1);
		}
	}
	
	/** function addTerm_toIndex
	 * 
	 * this function takes in a string representing a Term as input
	 * it then adds the URL to term_index (with a corresponding code)
	 * note: (term_index is a field in the myIndex)
	 * its simply a mapping of search terms to numbers (which are unique)
	 * 
	 * if the Term is already in the term_index, this function does nothing
	 * 
	 * 
	 * @param Term: the Term we want to store
	 * @param myIndex: a representation of the web database
	 * @return void: this function has no returns
	 */
	private void addTerm_toIndex(String Term, Index myIndex){
		
		HashMap<String, Integer> term_index = myIndex.getTerm_index();
		
		//if the index does not already contain the Term in question
		if(!term_index.containsKey(Term)){
			//add the URL to the index
			term_index.put(Term, term_index.size()+1);
		}
		
	}
	
	/** function createJedis
	 * 
	 *  this function creates a Jedis object that talks to
	 *  a prespecified database
	 * 
	 * @return Jedis: a Jedis object that talks to a prespecified database
	 * @throws IOException 
	 * @throws ClassNotFoundException 
	 */
	private Jedis createJedis()
			throws ClassNotFoundException, IOException{
		
		//this is just an instance of serial class
			//that will be used for making (de)serializing objects
			serial_class my_serial=new serial_class();
				
		//boilerplate jedis stuff
		String host = "soldierfish.redistogo.com";
		String auth = "c527e75f8f45baa5d34dd12b463c3f8b";
		int port = 9022;
		Jedis jedis = new Jedis(host, port);
		jedis.auth(auth);
		
		return jedis;
	}
	
	/** function storeObject
	 * 
	 * this function takes in a Serializable object as input
	 * as well as a label for the object
	 * 
	 * it then stores the serialization of the object under 
	 * the given label in a prespecified redis database
	 * 
	 * @param my_ser: the Serializable object we want to store
	 * @param label: the label that we want to store the object under
	 * @return void: this function has no returns
	 */
	private void storeObject(String label, Serializable my_ser)
			throws ClassNotFoundException, IOException{
		this.storeObject(label.getBytes(), my_ser);
	}
	

	/** function storeObject
	 * 
	 * this function takes in a Serializable object as input
	 * as well as a label for the object
	 * 
	 * it then stores the serialization of the object under 
	 * the given label in a prespecified redis database
	 * 
	 * @param my_ser: the Serializable object we want to store
	 * @param label: the label that we want to store the object under
	 * @return void: this function has no returns
	 * @throws IOException 
	 * @throws ClassNotFoundException 
	 */
	private void storeObject(byte [] label, Serializable my_ser)
			throws ClassNotFoundException, IOException{
		
		//this is just an instance of serial class
		//that will be used for making (de)serializing objects
		serial_class my_serial=new serial_class();
				
		//make the instance of Jedis that will talk to the
		//prespecified database
		Jedis jedis = this.createJedis();
		
		//store the serialization of the object under the label in the database
		jedis.set(label,my_serial.serialize(my_ser));
		
		jedis.disconnect();
	}
	
	
	/** function retrieveObject
	 * 
	 * this function takes in a String label
	 * 
	 * it then finds a object in a prespecified redis database
	 * that is stored under the aforemention label
	 * it then deserializes the object and returns the result
	 * 
	 * @param label: the label that the object we want to 
	 * retrieve is stored under
	 * @return Object: the object stored under label
	 * @throws IOException 
	 * @throws ClassNotFoundException 
	 */
	private Object retrieveObject(String label)
			throws ClassNotFoundException, IOException{
		return this.retrieveObject(label.getBytes());
	}
	
	/** function retrieveObject
	 * 
	 * this function takes in a byte array
	 * 
	 * it then finds a object in a prespecified redis database
	 * that is stored under the aforemention label
	 * it then deserializes the object and returns the result
	 * 
	 * @param label: the label that the object we want to 
	 * retrieve is stored under
	 * @return Object: the object stored under label
	 * @throws IOException 
	 * @throws ClassNotFoundException 
	 */
	private Object retrieveObject(byte[] label)
			throws ClassNotFoundException, IOException{
		
		//this is just an instance of serial class
		//that will be used for making (de)serializing objects
		serial_class my_serial=new serial_class();
		
		//make the instance of Jedis that will talk to the
		//prespecified database
		Jedis jedis = this.createJedis();
		
		//retrieve the serialization of the object that was stored under label
		byte[] yourbytes=jedis.get(label);
		
		jedis.disconnect();
		
		//then deserialize the object and return it.
		return my_serial.deserialize(yourbytes);
	}
	
	
}
